import sys
import re
from pyrem_torq.utility import split_to_strings_iter

if sys.platform == 'cli':
    import System.Text.Encoding as _ste
    _filesystemencoding = "%d" % _ste.Default.CodePage
else:
    _filesystemencoding = sys.getfilesystemencoding()

def get_version(): return (0, 0)
def get_option_description(): return []

def normalized_option_strs(options): return []
def get_target_file_predicate(options): return lambda path: False

def build_decoder(options): # return a function that may raise UnicodeDecodeError
    def dec(inputFileBytes, inputFilePath=None):
        return unicode(inputFileBytes, _filesystemencoding)
    return dec

def build_tokenizer(options):
    textSplitPattern = re.compile(r"\d+|[a-zA-Z]+|\r\n|.", re.DOTALL | re.IGNORECASE)
    def tkn(inputText, inputFilePath=None):
        seq = [ 'code' ]; seq.extend(split_to_strings_iter(inputText, textSplitPattern))
        seq.append([ 'eof' ]) # insert eof node at the end of sequence
        return seq
    return tkn

def build_whitelist(options): return []
def build_exprs(options): return []
def build_nodeformattable(options): return {}

